# -*- coding: utf-8 -*-
import scrapy
import codecs
import json
import MySQLdb
import MySQLdb.cursors

from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
from twisted.enterprise import adbapi
from scrapy.exceptions import DropItem


class ArticlespiderPipeline(object):
    @staticmethod
    def process_item(item, spider):
        return item


class MySqlTwistedPipeline(object):
    # 采用同步的机制写入mysql
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparms = dict(
            host=settings["MYSQL_HOST"],
            db=settings["MYSQL_DBNAME"],
            user=settings["MYSQL_USER"],
            password=settings["MYSQL_PASSWORD"],
            charset="utf8",
            cursorclass=MySQLdb.cursors.DictCursor, use_unicode=True)

        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        # 使用twisted将mysql插入变成异步执行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.hand_error, item, spider)

    # 处理异常
    @staticmethod
    def hand_error(failure, item, spider):
        # 处理异步插入的异常
        print(failure)

    @staticmethod
    def do_insert(cursor, item):
        # 执行具体的插入操作
        # 根据不同的item 构建不同的sql语句并插入到mysql中
        insert_sql, params = item.get_insert_sql()
        cursor.execute(insert_sql, params)


class MysqlPipeline(object):
    def __init__(self):
        self.conn = MySQLdb.connect(
            'localhost', 'root', 'root123', 'article_spider', charset="utf8", use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
                          insert into article(url_object_id,title,create_date,url,front_image_url,
                          fav_nums,article_text,read_volume)
                          values (%s,%s,%s,%s,%s,%s,%s)
                      """
        self.cursor.execute(insert_sql, (
            item["url_object_id"], item["title"], item["create_date"], item["url"],
            item["fav_nums"], item["article_text"], item["read_volume"]))
        self.conn.commit()


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                  '(KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
    'Referer': 'http://segmentfault.com/',
}


class ArticleImagePipeline(ImagesPipeline):
    # 重载ImagePipeline中的item_completed方法，获取下载地址
    def get_media_requests(self, item, info):
        for image_url in item['front_image_url']:
            yield scrapy.Request(url=image_url, meta={'item': item}, headers=headers)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        item['front_image_path'] = image_paths
        return item


class JsonExporterPipeline(object):
    # 调用scrapy提供的json exprot到处json文件
    def __init__(self):
        self.file = open('article_exporter.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding="utf-8", ensure_ascii=False)
        self.exporter.start_exporting()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

    def close_spider(self):
        self.exporter.finish_exporting()
        self.file.close()


class JsonWithEncodingPipeline(object):
    def __init__(self):
        self.file = codecs.open('article.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.close()


class ElasticsearchPipline(object):
    # 讲数据写入到es中
    @staticmethod
    def process_item(item, spider):
        item.save_to_es()
        return item
